package com.zlm.spider.test;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.seimicrawler.xpath.JXDocument;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;

/**
 * @author Harbour
 * @date 2021-04-12 14:04
 */
public class JsoupXPathTest51 {
    public static void main(String[] args) {

        CloseableHttpClient client = HttpClients.createDefault();

        try {
            String url = "https://search.51job.com/list/000000,000000,0000,32%252c01,9,99,+,2,2.html?" +
                    "lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99" +
                    "&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";

            WebClient wc = new WebClient(BrowserVersion.CHROME);
            wc.waitForBackgroundJavaScript(1000);
//            wc.setAjaxController(new NicelyResynchronizingAjaxController());

            HtmlPage page = wc.getPage(url);
            String pageXml = page.asXml();
            Document document1 = Jsoup.parse(pageXml);
//            System.out.println(document1);

            HttpGet httpGet = new HttpGet(url);
            // 必须要设置对象头，否则一直提示登录
            httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0");

            CloseableHttpResponse response = client.execute(httpGet);
            HttpEntity entity = response.getEntity();
            String content = EntityUtils.toString(entity, StandardCharsets.UTF_8);

            JXDocument document = JXDocument.create(Jsoup.parse(content));
            JXDocument document2 = JXDocument.create(document1);
            String xpath = "//*[@class=\"j_joblist\"]";
            Object o = document2.selOne(xpath);
            if (o instanceof Element){
                Element e = (Element) o;
                String s = e.text();
                String book = s.replace("<font class=\"skcolor_ljg\">书</font>", "书");
                System.out.println(book);
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
